package com.terren.spider.core.html.processor;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.collections.CollectionUtils;

import com.terren.spider.core.html.configuration.Configuration;
import com.terren.spider.core.html.scheme.ICustomScheme;
import com.terren.spider.entity.config.SysConfig;
import com.terren.spider.entity.core.Entry;
import com.terren.spider.entity.core.EntryFormat;
import com.terren.spider.entity.core.HelpUrl;
import com.terren.spider.entity.core.SelectorType;
import com.terren.spider.util.BasicUtil;
import com.terren.spider.util.common.BeanUtil;
import com.terren.spider.util.common.StringUtil;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.JsonPathSelector;

public abstract class BasicHtmlProcessor extends BasicUtil implements PageProcessor {
	protected Configuration configuration;// 配置项
	protected Set<String> helpUrls;// 列表页集合
	protected Set<String> targetUrls;// 目标页集合
	protected Set<String> helpPagingUrls;// 列表页的分页 连接集合
	protected Integer currentPageCount;// 当前页

	protected Map<String, String> headers = new HashMap<>();//header
	
	public BasicHtmlProcessor(Configuration configuration) throws Exception {
		this.configuration = configuration;
		init();
	}
	public void init() {
		helpUrls = new HashSet<String>();
		targetUrls = new HashSet<String>();
		helpPagingUrls = new HashSet<String>();
		currentPageCount = 0;
	}



	/**
	 * 添加列表页
	 * 
	 * @param page
	 * @param entry
	 */
	public List<String> findHelpUrls(Page page, Entry entry) {
		List<String> helpUrlList = new ArrayList<>();
		if (isContainHelpUrl(entry)) {
			for (HelpUrl helpUrl : entry.getHelpUrls()) {
				String select = helpUrl.getSelect();// 选择器 配置
				String urlRegex = helpUrl.getUrlRegex();// url正则
				List<String> findUrls = findUrls(page, select, SelectorType.valueOfType(helpUrl.getSelectorType()),
						urlRegex);
				if (findUrls.size() > 0) {
					findUrls = this.filterExcludeUrl(findUrls, helpUrl.getExcludeUrlRegex());// 过滤掉不需要的url
					findUrls = this.formatHelpUrl(findUrls, entry);// 格式化链接

					helpUrls.addAll(findUrls);
					helpUrls.remove(null);
					helpUrlList.addAll(helpUrls);
				}
			}
		}
		return helpUrlList;
	}

	/**
	 * 添加目标页
	 * 
	 * @param page
	 * @param entry
	 */
	public List<String> findTargetUrls(Page page, Entry entry) {
		// log.info("regex ------ >"+entry.getUrlRegex());
		// System.out.println(page.getHtml().links().all());
		String select = entry.getSelect();// 选择器 配置
		List<String> resultList = new ArrayList<>();
		List<String> allTarget = new ArrayList<String>();
		/**** entry 配置了UrlRegex 目标页正则表达式 ****/
		if (null != entry.getUrlRegex()) { // 目标正则不为空
			String[] urlRegexList = entry.getUrlRegex().split(SysConfig.URL_SPLIT);
			for (String url : urlRegexList) {
				List<String> findUrls = findUrls(page, select, SelectorType.valueOfType(entry.getSelectorType()), url);// 根据目标页正则
				allTarget.addAll(findUrls);
			}
		}
		/**** entry 没有配置UrlRegex 目标页正则表达式 ****/
		else {// 目标正则为空
			List<String> findUrls = findUrls(page, select, SelectorType.valueOfType(entry.getSelectorType()), null);// 根据目标页正则
			allTarget.addAll(findUrls);
		}
		/**** entry 配置了ExcludeUrlRegex 排除的目标页正则表达式 ****/
		if (allTarget.size() > 0) {
			allTarget = this.filterExcludeUrl(allTarget, entry.getExcludeUrlRegex());// 排除掉不要的链接
			allTarget = this.formatTargetUrl(allTarget, entry);// 格式化目标页链接 从`t_entry_format`读取配置
			targetUrls.addAll(allTarget);
			targetUrls.remove(null);
			resultList.addAll(targetUrls);
		}
		return resultList;
	}

	/**
	 * 列表页 的分页
	 * 
	 * @param page
	 * @param entry
	 * @return
	 */
	public List<String> findHelpPaginationUrl(Page page,Configuration configuration) {
		List<String> listUrl = new ArrayList<String>();
		List<String> resultList = new ArrayList<>();
		for (ICustomScheme scheme : configuration.getSchemes()) {// 分页[下一页]
			if (BeanUtil.includeMethod("findHelpPaginationUrl", scheme.getClass())) {
				log.info("[entry:"+configuration.getEntry().getEntryId()+"]"+"[scheme:"+scheme.getClass().getName());
				Set<String> urList = scheme.findHelpPaginationUrl(page, configuration.getEntry());
				log.info("[method:findHelpPaginationUrl]"+"[size:"+urList.size()+"]");
				if (null != urList) {
					listUrl.addAll(urList);
				}
			}
		}
		if (listUrl.size() > 0) {
			helpPagingUrls.addAll(listUrl);
			helpPagingUrls.remove(null);
			resultList.addAll(helpPagingUrls);
		}
		return resultList;
	}
	/**
	 * 目标页 的分页
	 * 
	 * @param page
	 * @param entry
	 * @return
	 */
	public List<String> findTargetPaginationUrl(Page page,Configuration configuration) {
		List<String> listUrl = new ArrayList<String>();
		for (ICustomScheme scheme : configuration.getSchemes()) {// 分页[下一页]
			if (BeanUtil.includeMethod("findTargetPaginationUrl", scheme.getClass())) {
				log.info("[entry:"+configuration.getEntry().getEntryId()+"]"+"[scheme:"+scheme.getClass().getName());
				Set<String> urList = scheme.findTargetPaginationUrl(page, configuration.getEntry());
				log.info("[method:findTargetPaginationUrl]"+"[size:"+urList.size()+"]");
				if (null != urList) {
					listUrl.addAll(urList);
				}
			}
		}
		return listUrl;
	}

	/**
	 * 是否包含列表页
	 * 
	 * @param entry
	 * @return
	 */
	public Boolean isContainHelpUrl(Entry entry) {
		return (null != entry.getHelpUrls() && !entry.getHelpUrls().isEmpty());
	}

	/**
	 * 是否是入口页面
	 * 
	 * @param page
	 * @param entry
	 * @return
	 */
	public Boolean isEntryUrl(Page page) {
		String url = getCurrentPageUrl(page);
		return configuration.getEntry().getEntryUrl().equals(url);
	}

	/**
	 * 页面是否为目标页
	 * 
	 * @param page
	 * @param entry
	 * @return
	 */
	public Boolean isTargetUrl(Page page, Entry entry) {
		String url = getCurrentPageUrl(page);
		if (null != entry.getUrlRegex() && !"".equals(entry.getUrlRegex())) {
			String[] urlRegexList = entry.getUrlRegex().split(SysConfig.URL_SPLIT);
			for (String urlreg : urlRegexList) {
				if (StringUtil.match(url, urlreg)) {
					return true;
				}
			}
		}
		log.info("it is not targetUrl["+url+"] ");
		return false;
	}

	/**
	 * 页面是否为列表页
	 * 
	 * @param page
	 * @param entry
	 * @return
	 */
	public Boolean isHelpUrl(Page page, Entry entry) {
		String url = getCurrentPageUrl(page);
		if (isContainHelpUrl(entry)) {
			for (HelpUrl helpUrl : entry.getHelpUrls()) {
				if (StringUtil.match(url, helpUrl.getUrlRegex())) {
					return true;
				}

			}
		}
		return false;
	}

	/**
	 * 得到当前页面的链接 如果使用代理 webmagic取不到正确域名 所以需要处理
	 * 
	 * @param page
	 * @param entry
	 * @param useProxy
	 * @return
	 */
	/*public String getCurrentPageUrl(Page page,Boolean useProxy) {
		String url = null;
		if (useProxy) {// 使用代理
			url = (String) page.getRequest().getExtra("url");
		} else {
			url = page.getUrl().toString();
		}
		return url;
	}*/
	
	public String getCurrentPageUrl(Page page) {
		return page.getUrl().toString();
	}

	/**
	 * 页面是否为排除的目标页
	 * 
	 * @param page
	 * @param entry
	 * @return
	 */
	public Boolean isExcludeTargetUrl(Page page, Entry entry) {
		String url = getCurrentPageUrl(page);
		if (null != entry.getExcludeUrlRegex() && !"".equals(entry.getExcludeUrlRegex())) {
			String[] urlRegexList = entry.getExcludeUrlRegex().split(SysConfig.URL_SPLIT);
			for (String regexUrl : urlRegexList) {
				if (StringUtil.match(url, regexUrl)) {
					return true;
				}
			}
		}
		return false;
	}

	/**
	 * 过滤掉不匹配的url
	 * 
	 * @param urlList
	 * @param excludeUrl
	 * @return
	 */
	public List<String> filterExcludeUrl(List<String> urlList, String excludeUrl) {

		if (null == excludeUrl || "".equals(excludeUrl)) {
			return urlList;
		} else {// 如果有排除regex，则过滤掉不要的url
			List<String> filterList = new ArrayList<>();
			if (!urlList.isEmpty()) {
				for (String url : urlList) {
					boolean flag = true;
					if (null != excludeUrl && !"".equals(excludeUrl)) {
						String[] urlRegexList = excludeUrl.split(SysConfig.URL_SPLIT);
						for (String regexUrl : urlRegexList) {
							if (StringUtil.match(url, regexUrl) || "".equals(url)) {// 不匹配排除的url ，添加到列表
								flag = false;
								break;
							}
						}
						if (flag) {
							filterList.add(url);
						}

					}
				}
			}
			return filterList;
		}

	}

	/**
	 * 
	 * @param page
	 * @param select
	 * @param selectorType
	 * @param urlRegex
	 * @return
	 */
	private List<String> findUrls(Page page, String select, SelectorType selectorType, String urlRegex) {
		List<String> findUrls = new ArrayList<String>();
		/**
		 * 如果 select 不为空 则表示抓取某个div下的链接
		 */

		// log.info(page.getHtml().links().all());
		if (select != null && !select.equals("")) {
			switch (selectorType) {// 抓取页面 选择器选择 CSS (0) | XPATH (1) | JSONPATH(2)
			case CSS:
				if (null != urlRegex && !urlRegex.equals("")) {
					findUrls.addAll(page.getHtml().css(select).links().regex(urlRegex).all());
				} else {
					findUrls = page.getHtml().css(select).links().all();
				}
				break;
			case XPATH:
				if (null != urlRegex && !urlRegex.equals("")) {
					findUrls.addAll(page.getHtml().xpath(select).links().regex(urlRegex).all());
				} else {
					findUrls.addAll(page.getHtml().xpath(select).links().all());
				}

				break;
			case JSON_PATH:
				List<String> jsonUrls = new JsonPathSelector(select).selectList(page.getRawText());// 例 ：
																											// $.data[*]._id
				findUrls.addAll(jsonUrls);
				break;
			default:
				break;
			}
		}
		/**
		 * 如果 select 为空 则表示抓取所有链接
		 */
		else {
			if (null != urlRegex && !urlRegex.equals("")) {
				findUrls = page.getHtml().links().regex(urlRegex).all();
			}
		}
		return findUrls;
	}

	/**
	 * 
	 * @param page
	 * @param urls
	 */
	/*@Deprecated
	public void addHelpUrlTargetRequests(Page page, Set<String> urls) {
		if (null != urls) {
			urls.remove(null);
			Request[] requests = HttpUtil.getRequests(configuration.getProxyRequestParam().getEntryUseProxy(), entry,
					urls.toArray(new String[urls.size()]));
			for (Request request : requests) {
				page.addTargetRequest(request);
			}
		}
	}*/

	
	public void addTargetRequests(Page page, Set<String> urls) {
		if (null != urls&&urls.size()>0) {
			urls.remove(null);
			List<String> ulist = new ArrayList<>();
			ulist.addAll(urls);
			page.addTargetRequests(ulist);
		}
	}
	
	// /**
	// * 方法是为了解决 webmagic 的bug，把链接的domain替换成原始路径
	// *
	// * @param url
	// * @return
	// */
	// protected String formatProxyUrl(String url) {
	// if (null != url) {
	// Pattern patternForProtocal = Pattern.compile("://.+");
	// String prex = patternForProtocal.matcher(url).replaceAll("");// http or https
	// String replace_host = prex + "://" + configuration.getDomain() + "/";
	// // http://192.168.20.8:13003/
	// String proxyServer = "http://" + entry.getProxyHost() + ":" +
	// entry.getProxyPort() + "/";
	// return StringUtils.replace(url, proxyServer, replace_host);
	// }
	// return null;
	// }

	/**
	 * 方法是为了解决 webmagic 的bug，把链接的domain替换成原始路径
	 * 
	 * @param url
	 * @return
	 *//*
	@Deprecated
	protected List<String> formatProxyUrl(List<String> urls) {
		List<String> resultUrls = new ArrayList<>();
		if (CollectionUtils.isNotEmpty(urls)) {
			for (String url : urls) {
				Pattern patternForProtocal = Pattern.compile("://.+");
				String prex = patternForProtocal.matcher(url).replaceAll("");// http or https
				String replace_host = prex + "://" + configuration.getDomain() + "/";
				// http://192.168.20.8:13003/
				String proxyServer = "http://" + entry.getProxyHost() + ":" + entry.getProxyPort() + "/";
				resultUrls.add(StringUtils.replace(url, proxyServer, replace_host));
			}
		}
		return resultUrls;
	}*/

	/**
	 * 格式化 url
	 * 
	 * @param urls
	 * @param url_format
	 * @return
	 */
	protected List<String> formatUrl(List<String> urls, String url_format) {
		if (url_format == null||"".equals(url_format)) {
			return urls;
		}
		List<String> resultUrls = new ArrayList<>();
		if (CollectionUtils.isNotEmpty(urls)) {
			for (String id : urls) {
				Object[] args = { id };
				String url = StringUtil.messageFormat(url_format,args);
				resultUrls.add(url);
			}
		}
		return resultUrls;
	}

	/**
	 * 格式化列表页链接 从`t_entry_format`读取配置
	 * 
	 * @param urls
	 * @param entry
	 * @return
	 */
	protected List<String> formatHelpUrl(List<String> urls, Entry entry) {
		List<String> resultUrls = new ArrayList<>();
		Set<String> urlset = new HashSet<>();
		if (CollectionUtils.isNotEmpty(entry.getFormats())) {
			for (EntryFormat format : entry.getFormats()) {
				String url_format = format.getHelpurlFormat();
				if (null!=url_format&&!"".equals(url_format)) {
					urlset.addAll(formatUrl(urls, url_format));
				}else {
					urlset.addAll(urls);
				}
			}
		}else {
			urlset.addAll(urls);
		}
		resultUrls.addAll(urlset);
		return resultUrls;
		
	}

	/**
	 * 格式化目标页链接 从`t_entry_format`读取配置
	 * 
	 * @param urls
	 * @param entry
	 * @return
	 */
	protected List<String> formatTargetUrl(List<String> urls, Entry entry) {
		List<String> resultUrls = new ArrayList<>();
		Set<String> urlset = new HashSet<>();
		if (CollectionUtils.isNotEmpty(entry.getFormats())) {
			for (EntryFormat format : entry.getFormats()) {
				String url_format = format.getTargeturlFormat();
				if (null!=url_format&&!"".equals(url_format)) {
					urlset.addAll(formatUrl(urls, url_format));
				}else {
					urlset.addAll(urls);
				}
			}
		}else {
			urlset.addAll(urls);
		}
		resultUrls.addAll(urlset);
		return resultUrls;
	}

	@Override
	public Site getSite() {
		Site site = Site.me();
		if (null == configuration) {
			configuration = new Configuration();
		}
		if (null != configuration.getSleepTime()) {
			site.setSleepTime(configuration.getSleepTime());
		}
		if (null != configuration.getRetryTimes()) {
			site.setRetryTimes(configuration.getRetryTimes());
		}
		if (null != configuration.getCharset()) {
			site.setCharset(configuration.getCharset());
		}
		if (null != configuration.getCycleRetryTimes()) {
			site.setCycleRetryTimes(configuration.getCycleRetryTimes());
		}
		if (null != configuration.getAcceptStatCode()) {
			site.setAcceptStatCode(configuration.getAcceptStatCode());
		}
		if (null != configuration.getDisableCookieManagement()) {
			site.setDisableCookieManagement(configuration.getDisableCookieManagement());
		}
		if (null != configuration.getDomain()) {
			site.setDomain(configuration.getDomain());
		}
		if (null != configuration.getRetrySleepTime()) {
			site.setRetrySleepTime(configuration.getRetrySleepTime());
		}
		if (null != configuration.getTimeOut()) {
			site.setTimeOut(configuration.getTimeOut());
		}
		if (null != configuration.getUseGzip()) {
			site.setUseGzip(configuration.getUseGzip());
		}
		if (null != configuration.getUserAgent()) {
			site.setUserAgent(configuration.getUserAgent());
		}
		if (null!=configuration.getCookies()) {
			for (Map.Entry<String, String> cookieEntry : configuration.getCookies().entrySet()) {
				site.addCookie(cookieEntry.getKey(), cookieEntry.getValue());
            } 
		}
		if (null!=configuration.getHeaders()) {
			for (Map.Entry<String, String> headerEntry : configuration.getHeaders().entrySet()) {
				site.addHeader(headerEntry.getKey(), headerEntry.getValue());
            }
		}
		
		return site;
	}


	public Configuration getConfiguration() {
		return configuration;
	}

	public void setConfiguration(Configuration configuration) {
		this.configuration = configuration;
	}

	public Set<String> getHelpUrls() {
		return helpUrls;
	}

	public void setHelpUrls(Set<String> helpUrls) {
		this.helpUrls = helpUrls;
	}

	public Set<String> getTargetUrls() {
		if (null != targetUrls) {
			targetUrls.remove(null);
		}
		return targetUrls;
	}

	public void setTargetUrls(Set<String> targetUrls) {
		this.targetUrls = targetUrls;
	}

	public Integer getTargetUrlCount() {
		if (null != targetUrls) {
			targetUrls.remove(null);
			return targetUrls.size();
		}
		return 0;
	}

	public Integer getHelpUrlCount() {
		if (null != helpUrls) {
			helpUrls.remove(null);
			return helpUrls.size();
		}
		return 0;
	}

	public Set<String> getHelpPagingUrls() {
		return helpPagingUrls;
	}

	public void setHelpPagingUrls(Set<String> helpPagingUrls) {
		this.helpPagingUrls = helpPagingUrls;
	}

	public Integer getHelpPagingUrlCount() {
		if (null != helpPagingUrls) {
			return helpPagingUrls.size();
		}
		return 0;
	}

	protected void logMessage(Page page) {
		log.info("[entry id:" + configuration.getEntry().getEntryId() + "]" + "[current :" + (++currentPageCount) + "]" + "[helpUrl size:"
				+ (helpUrls.size()) + "]" + "[targetUrl size:" + (targetUrls.size()) + "]");
		// log.info("[current :" + (++currentPageCount) + "]");
		// log.info("[helpUrl size:" + (helpUrls.size()) + "]");
		// log.info("[targetUrl size:" + (targetUrls.size()) + "]");
		// log.info("[config:" + getSite().toString() + "]");

		if (configuration.getProxyRequestParam().getEntryUseProxy() || configuration.getProxyRequestParam().getContentUseProxy()) {
			log.info("[targetUrl:" + page.getRequest().getExtra("url") + "]");
		}
	}
	
	protected void setHeader(Page page) {
		// TODO Auto-generated method stub
		Map<String, List<String>> responseHeaders = page.getHeaders();
		if (null!= responseHeaders) {
			for (Map.Entry<String, List<String>> headerEntry : responseHeaders.entrySet()) {
				String key = headerEntry.getKey();
				List<String> values = headerEntry.getValue();
				String strValue = "";
				for (String string : values) {
					strValue = strValue + string;
				}
				headers.put(key, strValue);
			}
		}
	}
	public Map<String, String> getHeaders() {
		return headers;
	}
}
